{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "535adad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.keyedvectors import KeyedVectors\n",
    "from gensim.models.fasttext import FastText\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "48ad3d4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vocab(text, min_freq=10):\n",
    "    vo = ' '.join(text).split(' ')\n",
    "    word_dict = {k:vo.count(k) for k in vo}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a3a3ccdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './Datasets/raw_chat_corpus/raw_chat_corpus/weibo-400w/stc_weibo_train_response'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8b50cf9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path, 'r', encoding='utf-8') as fp:\n",
    "    raw_texts = [i.rstrip('\\n') for i in fp.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7504a6cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "sep_text = ' '.join(raw_texts).split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b65a4b51",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_dict = {k:sep_text.count(k) for k in sep_text}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90a23e14",
   "metadata": {},
   "outputs": [],
   "source": [
    "[i.split('\\n') for i in sep_text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a440a7d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ec5c7f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = texts.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "fdfaa51f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4435960"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([i.split(' ') for i in texts])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5204cbff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Applications\\Miniconda\\envs\\nlp\\lib\\site-packages\\ipykernel_launcher.py:1: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "arr = np.asarray([i.split(' ') for i in texts])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9f8c8803",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([list(['王', '大姐', '，', '打字', '细心', '一点']),\n",
       "       list(['于', '老师', '不', '给', '劝劝', '架么', '告诉', '他们', '再', '挣', '也', '不', '是', '老大']),\n",
       "       list(['真不愧是', '这么', '走', '出来', '的', '爹·······']), ...,\n",
       "       list(['你们', '都', '是', '抢镜', '的', '高手', '！']),\n",
       "       list(['宝贝', '，', '一起', '碎觉觉', '吧', '，', '么', '！', '晚安', '。']),\n",
       "       list([''])], dtype=object)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "33a5ec68",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vec = KeyedVectors.load_word2vec_format('./models/cc.zh.300.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "45eb83a7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('杜兰特', 0.6702494025230408),\n",
       " ('麦蒂', 0.6513450145721436),\n",
       " ('哈登', 0.6310815215110779),\n",
       " ('德罗赞', 0.629206120967865),\n",
       " ('姚明', 0.6284971237182617),\n",
       " ('艾弗森', 0.6178681254386902),\n",
       " ('乔丹', 0.6172911524772644),\n",
       " ('皮蓬', 0.6147411465644836),\n",
       " ('威少', 0.6127848625183105),\n",
       " ('湖人队', 0.6122423410415649)]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_vec.most_similar(positive=['科比'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6746b095",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = FastText()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
